/**
 * @Title: HtmlScrapy
 * @Package com.xyl.test.okhttp
 * @Description: TODO
 * @author yulin.xu@downjoy.com
 * @date 2017/6/27 9:48
 * @version V1.0
 */
package com.xyl.test.okhttp;

import okhttp3.Call;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

/**
 *
 *@ClassName: HtmlScrapy
 * @Description: TODO
 * @author yulin.xu@downjoy.com
 * @date 2017/6/27 9:48
 *
 */
public class HtmlScrapy {

    private String saveDirPath;
    private String rootUrl;
    private OkHttpClient httpClient;
    private Map<String, String> urlMap;

    public HtmlScrapy(String rootUrl, String saveDirPath) {
        this.rootUrl = rootUrl;
        this.saveDirPath = saveDirPath;
        init();
    }

    private void init() {
        httpClient = new OkHttpClient();
        urlMap = new HashMap<String, String>();
    }

    private Map<String, Object> httpGet(String url) {
        Map<String, Object> result = new HashMap<String, Object>();
        Request request = new Request.Builder().url(url).build();
        Call call = httpClient.newCall(request);
        try {
            Response response = call.execute();
            int code = response.code();
            result.put("code", code);
            if (code == 200) {
                result.put("data", response.body().bytes());
            }
            response.close();
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
        return result;
    }

    private void parsePageUrls(String text, String prefix, String dir) {
        if (!"".equals(text)) {
            int hrefIndex = text.indexOf(prefix, 0);
            String reset = text.substring(hrefIndex + prefix.length());
            while (hrefIndex != -1) {
                String s = "'";
                if (reset.startsWith("\"")) {
                    s = "\"";
                } else if (reset.startsWith("(")) {
                    s = "(";
                }
                if (reset.indexOf(s, 1) == -1) {
                    return;
                }
                if (s.equals("(")) {
                    s = ")";
                }
                String url = reset.substring(1, reset.indexOf(s, 1));
                urlMap.put(url, dir);
                hrefIndex = reset.indexOf(prefix, 0);
                reset = reset.substring(hrefIndex + prefix.length());
            }
        }
    }

    private boolean exists(String url, String dir) {
        String path = saveDirPath;
        if (!"".equals(dir)) {
            path = saveDirPath + "\\" + dir.replace("/", "\\") + "\\";
        }
        File file = new File(path);
        if (!file.exists()) {
            file.mkdir();
        }
        if (url.startsWith("../../../")) {
            file = file.getParentFile().getParentFile().getParentFile();
            if (!file.exists()) {
                file.mkdir();
            }
        } else if (url.startsWith("../../")) {
            file = file.getParentFile().getParentFile();
            if (!file.exists()) {
                file.mkdir();
            }
        } else if (url.startsWith("../")) {
            file = file.getParentFile();
            if (!file.exists()) {
                file.mkdir();
            }
        }
        String fileName = file.getPath();
        if (rootUrl.equals(url)) {
            fileName += "\\index.html";
        } else if (!url.replace("../", "").contains(".")) {
            if (url.equals(dir)) {
                fileName += "\\index.html";
            } else {
                fileName += "\\" + url.replace("../", "").replace("/", "\\") + "\\index.html";
            }
        } else {
            fileName += "\\" + url.replace("../", "").replace("/", "\\");
        }
        File newFile = new File(fileName);

        String[] arr = newFile.getPath().split("\\\\");
        String tmp = "";
        for (int i = 0; i < arr.length-1; i++) {
            if (!"".equals(tmp)) {
                tmp += "\\";
            }
            tmp += arr[i];
            File file1 = new File(tmp);
            if (!file1.exists()) {
                file1.mkdir();
            }
        }

        if (!newFile.exists()) {
            return false;
        } else {
            return true;
        }
    }

    private boolean writeFile(String url, String dir, byte[] text) {
        String path = saveDirPath;
        if (!"".equals(dir)) {
            path = saveDirPath + "\\" + dir.replace("/", "\\") + "\\";
        }
        File file = new File(path);
        if (!file.exists()) {
            file.mkdir();
        }
        if (url.startsWith("../../../")) {
            file = file.getParentFile().getParentFile().getParentFile();
            if (!file.exists()) {
                file.mkdir();
            }
        } else if (url.startsWith("../../")) {
            file = file.getParentFile().getParentFile();
            if (!file.exists()) {
                file.mkdir();
            }
        } else if (url.startsWith("../")) {
            file = file.getParentFile();
            if (!file.exists()) {
                file.mkdir();
            }
        }
        String fileName = file.getPath();
        if (rootUrl.equals(url)) {
            fileName += "\\index.html";
        } else if (!url.replace("../", "").contains(".")) {
            if (url.equals(dir)) {
                fileName += "\\index.html";
            } else {
                fileName += "\\" + url.replace("../", "").replace("/", "\\") + "\\index.html";
            }
        } else {
            fileName += "\\" + url.replace("../", "").replace("/", "\\");
        }
        File newFile = new File(fileName);

       String[] arr = newFile.getPath().split("\\\\");
       String tmp = "";
        for (int i = 0; i < arr.length-1; i++) {
            if (!"".equals(tmp)) {
                tmp += "\\";
            }
            tmp += arr[i];
            File file1 = new File(tmp);
            if (!file1.exists()) {
                file1.mkdir();
            }
        }

        if (!newFile.exists()) {
            try {
                newFile.createNewFile();
                FileOutputStream fileOutputStream = new FileOutputStream(newFile);
                fileOutputStream.write(text);
                fileOutputStream.close();
            } catch (IOException e) {
                return false;
            }
        }
        return true;
    }

    public void start() throws Exception {
        File file = new File(saveDirPath);
        if (!file.exists()) {
            throw new Exception(saveDirPath + " not exists");
        }
        if (!file.isDirectory()) {
            throw new Exception(saveDirPath + " is not directory");
        }
        Map<String, Object> result = httpGet(rootUrl);
        byte[] rootPageBytes = (byte[]) result.get("data");
        writeFile(rootUrl, "", rootPageBytes);
        String rootPage = new String(rootPageBytes);
        if (null == rootPage) {
            throw new Exception(rootUrl + " 网络异常");
        }
        parsePageUrls(rootPage, "href=", "");
        parsePageUrls(rootPage, "src=", "");
        parsePageUrls(rootPage, "image: ", "");

        String page = "";
        String url = "";
        String dir = "";
        while (urlMap.entrySet().iterator().hasNext()) {
            Map.Entry<String, String> entry = urlMap.entrySet().iterator().next();
            url = entry.getKey();
            if (!"".equals(url)) {
                dir = entry.getValue();
                if (!url.contains(".")) {
                    dir = url;
                }
                if (url.startsWith("../")) {
                    url = url.replace("../", "");
                }
                if ("online-application".equals(dir)) {
                    url = "online-application/" + url;
                }
                if (!url.startsWith("http")) {
                    url = rootUrl + url;
                }
                if (!"".equals(entry.getKey().replace("../", ""))) {
                    if (!(entry.getKey().startsWith("http") || entry.getKey().startsWith("//") ||
                            entry.getKey().contains("+img_src+") || entry.getKey().contains("#'+ nr +'")
                            || entry.getKey().startsWith("#") || entry.getKey().startsWith("his.status.src;")
                            || entry.getKey().startsWith("p_images[set_position+1]") || entry.getKey().startsWith("[d];c.status.format[d]")
                            || entry.getKey().startsWith("location_href}") || entry.getKey().startsWith("about:blank")
                            || entry.getKey().startsWith("+f+") || entry.getKey().startsWith("'+ thumb +'")
                            || entry.getKey().startsWith("+\"://f.vimeocdn.com") || entry.getKey().startsWith("'+f+'")
                            || entry.getKey().startsWith("[e],c.status.format") || entry.getKey().startsWith("=t&&(n=!0)}),!n){var")
                            || entry.getKey().startsWith("'+b+") || entry.getKey().startsWith(".src,i.progress=\"inload\"}})}")
                            || entry.getKey().length() > 200 || entry.getKey().startsWith("'") || entry.getKey().startsWith("{")
                            || entry.getKey().startsWith("p_images[n]}toInject+=\"<l") || entry.getKey().startsWith(".src;return e(t)")
                            || entry.getKey().startsWith(",") || entry.getKey().startsWith("unction(h){return") || entry.getKey().startsWith("mailto")
                            || entry.getKey().startsWith("\" + image + \"") || entry.getKey().startsWith("+44.1223783290")
                            || entry.getKey().equals("/a") || entry.getKey().startsWith("\"") || entry.getKey().startsWith(" codecs=\"avc1.42E01"))) {
                        System.out.println(String.format("url:%s, dir:%s", entry.getKey(), dir));
                        if (entry.getKey().contains("split(")) {
                            System.out.println();
                        }
                        result = httpGet(url);
                        int code = (Integer)result.get("code");
                        if (code == 200) {
                            byte[] pageBytes = (byte[]) result.get("data");
                            if (!exists(entry.getKey(), dir)) {
                                boolean success = writeFile(entry.getKey(), dir, pageBytes);
                                if (success) {
                                    if (!entry.getKey().endsWith(".jpg") && !entry.getKey().startsWith("mailto")) {
                                        page = new String(pageBytes);
                                        if (!entry.getKey().replace("../", "").contains(".")) {
                                            dir = entry.getKey().replace("../", "");
                                        }
                                        parsePageUrls(page, "href=", dir);
                                        parsePageUrls(page, "src=", dir);
                                        parsePageUrls(page, "url", dir);
                                    }
                                }
                            }
                        }
                    }
                }
            }
            urlMap.remove(entry.getKey());
        }

    }

    public static void main(String[] args) throws Exception {

        HtmlScrapy htmlScrapy = new HtmlScrapy("http://www.steinsggs.com/", "D:\\xuyulin\\yang");
        htmlScrapy.start();

    }

}
